{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tutorial 6: Custom pipeline tutorial"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"../../imgs/lightautoml_logo_color.png\" alt=\"LightAutoML logo\" style=\"width:100%;\"/>"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Official LightAutoML github repository is [here](https://github.com/sb-ai-lab/LightAutoML)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1. Install LightAutoML"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Uncomment if doesn't clone repository by git. (ex.: colab, kaggle version)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! pip install -U lightautoml"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2. Import necessary libraries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Standard python libraries\n",
    "import os\n",
    "import time\n",
    "import requests\n",
    "\n",
    "\n",
    "# Installed libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "import torch\n",
    "\n",
    "# Imports from our package\n",
    "from lightautoml.automl.base import AutoML\n",
    "from lightautoml.ml_algo.boost_lgbm import BoostLGBM\n",
    "from lightautoml.ml_algo.tuning.optuna import OptunaTuner\n",
    "from lightautoml.pipelines.features.lgb_pipeline import LGBSimpleFeatures\n",
    "from lightautoml.pipelines.ml.base import MLPipeline\n",
    "from lightautoml.pipelines.selection.importance_based import ImportanceCutoffSelector, ModelBasedImportanceEstimator\n",
    "from lightautoml.reader.base import PandasToPandasReader\n",
    "from lightautoml.tasks import Task\n",
    "from lightautoml.automl.blend import WeightedBlender"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3. Parameters "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "N_THREADS = 8 # threads cnt for lgbm and linear models\n",
    "N_FOLDS = 5 # folds cnt for AutoML\n",
    "RANDOM_STATE = 42 # fixed random state for various reasons\n",
    "TEST_SIZE = 0.2 # Test size for metric check\n",
    "TARGET_NAME = 'TARGET' # Target column name"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4. Fix torch number of threads and numpy seed "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.random.seed(RANDOM_STATE)\n",
    "torch.set_num_threads(N_THREADS)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 5. Example data load "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load a dataset from the repository if doesn't clone repository by git."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "DATASET_DIR = '../data/'\n",
    "DATASET_NAME = 'sampled_app_train.csv'\n",
    "DATASET_FULLNAME = os.path.join(DATASET_DIR, DATASET_NAME)\n",
    "DATASET_URL = 'https://raw.githubusercontent.com/sb-ai-lab/LightAutoML/master/examples/data/sampled_app_train.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 28 µs, sys: 20 µs, total: 48 µs\n",
      "Wall time: 64.4 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "if not os.path.exists(DATASET_FULLNAME):\n",
    "    os.makedirs(DATASET_DIR, exist_ok=True)\n",
    "\n",
    "    dataset = requests.get(DATASET_URL).text\n",
    "    with open(DATASET_FULLNAME, 'w') as output:\n",
    "        output.write(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 105 ms, sys: 14.5 ms, total: 119 ms\n",
      "Wall time: 118 ms\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>...</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>313802</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>327024.0</td>\n",
       "      <td>15372.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>319656</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>108000.0</td>\n",
       "      <td>675000.0</td>\n",
       "      <td>19737.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>207678</td>\n",
       "      <td>0</td>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>F</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>2</td>\n",
       "      <td>112500.0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>13500.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>381593</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>1</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>142200.0</td>\n",
       "      <td>9630.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>258153</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>337500.0</td>\n",
       "      <td>1483231.5</td>\n",
       "      <td>46570.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR  \\\n",
       "0      313802       0         Cash loans           M            N   \n",
       "1      319656       0         Cash loans           F            N   \n",
       "2      207678       0    Revolving loans           F            Y   \n",
       "3      381593       0         Cash loans           F            N   \n",
       "4      258153       0         Cash loans           F            Y   \n",
       "\n",
       "  FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  \\\n",
       "0               Y             0          270000.0    327024.0      15372.0   \n",
       "1               N             0          108000.0    675000.0      19737.0   \n",
       "2               Y             2          112500.0    270000.0      13500.0   \n",
       "3               N             1           67500.0    142200.0       9630.0   \n",
       "4               Y             0          337500.0   1483231.5      46570.5   \n",
       "\n",
       "   ...  FLAG_DOCUMENT_18 FLAG_DOCUMENT_19 FLAG_DOCUMENT_20 FLAG_DOCUMENT_21  \\\n",
       "0  ...                 0                0                0                0   \n",
       "1  ...                 0                0                0                0   \n",
       "2  ...                 0                0                0                0   \n",
       "3  ...                 0                0                0                0   \n",
       "4  ...                 0                0                0                0   \n",
       "\n",
       "  AMT_REQ_CREDIT_BUREAU_HOUR AMT_REQ_CREDIT_BUREAU_DAY  \\\n",
       "0                        0.0                       0.0   \n",
       "1                        0.0                       0.0   \n",
       "2                        0.0                       0.0   \n",
       "3                        0.0                       0.0   \n",
       "4                        0.0                       0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_WEEK  AMT_REQ_CREDIT_BUREAU_MON  \\\n",
       "0                         0.0                        0.0   \n",
       "1                         0.0                        0.0   \n",
       "2                         0.0                        0.0   \n",
       "3                         0.0                        0.0   \n",
       "4                         0.0                        2.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_QRT  AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0                        0.0                         1.0  \n",
       "1                        0.0                         0.0  \n",
       "2                        0.0                         1.0  \n",
       "3                        0.0                         4.0  \n",
       "4                        0.0                         0.0  \n",
       "\n",
       "[5 rows x 122 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "data = pd.read_csv(DATASET_FULLNAME)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 6. (Optional) Some user feature preparation "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Cell below shows some user feature preparations to create task more difficult (this block can be omitted if you don't want to change the initial data):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 108 ms, sys: 4.5 ms, total: 113 ms\n",
      "Wall time: 111 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "data['BIRTH_DATE'] = (np.datetime64('2018-01-01') + data['DAYS_BIRTH'].astype(np.dtype('timedelta64[D]'))).astype(str)\n",
    "data['EMP_DATE'] = (np.datetime64('2018-01-01') + np.clip(data['DAYS_EMPLOYED'], None, 0).astype(np.dtype('timedelta64[D]'))\n",
    "                    ).astype(str)\n",
    "\n",
    "data['constant'] = 1\n",
    "data['allnan'] = np.nan\n",
    "\n",
    "data['report_dt'] = np.datetime64('2018-01-01')\n",
    "\n",
    "data.drop(['DAYS_BIRTH', 'DAYS_EMPLOYED'], axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 7. (Optional) Data splitting for train-test "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Block below can be omitted if you are going to train model only or you have specific train and test files:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data splitted. Parts sizes: train_data = (8000, 125), test_data = (2000, 125)\n",
      "CPU times: user 7.85 ms, sys: 3.89 ms, total: 11.7 ms\n",
      "Wall time: 10.1 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "train_data, test_data = train_test_split(data, \n",
    "                                         test_size=TEST_SIZE, \n",
    "                                         stratify=data[TARGET_NAME], \n",
    "                                         random_state=RANDOM_STATE)\n",
    "print('Data splitted. Parts sizes: train_data = {}, test_data = {}'\n",
    "              .format(train_data.shape, test_data.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>...</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "      <th>BIRTH_DATE</th>\n",
       "      <th>EMP_DATE</th>\n",
       "      <th>constant</th>\n",
       "      <th>allnan</th>\n",
       "      <th>report_dt</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>6444</th>\n",
       "      <td>112261</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>1</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>640080.0</td>\n",
       "      <td>31261.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1985-06-28</td>\n",
       "      <td>2012-06-21</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3586</th>\n",
       "      <td>115058</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>180000.0</td>\n",
       "      <td>239850.0</td>\n",
       "      <td>23850.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1953-12-27</td>\n",
       "      <td>2018-01-01</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9349</th>\n",
       "      <td>326623</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>112500.0</td>\n",
       "      <td>337500.0</td>\n",
       "      <td>31086.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1975-06-21</td>\n",
       "      <td>2016-06-17</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7734</th>\n",
       "      <td>191976</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>1</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>9018.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1988-04-27</td>\n",
       "      <td>2009-06-05</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-01-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2174</th>\n",
       "      <td>281519</td>\n",
       "      <td>0</td>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>10125.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1975-06-13</td>\n",
       "      <td>1997-01-22</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2018-01-01</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 125 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      SK_ID_CURR  TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR  \\\n",
       "6444      112261       0         Cash loans           F            N   \n",
       "3586      115058       0         Cash loans           F            N   \n",
       "9349      326623       0         Cash loans           F            N   \n",
       "7734      191976       0         Cash loans           M            Y   \n",
       "2174      281519       0    Revolving loans           F            N   \n",
       "\n",
       "     FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  \\\n",
       "6444               N             1           90000.0    640080.0      31261.5   \n",
       "3586               Y             0          180000.0    239850.0      23850.0   \n",
       "9349               Y             0          112500.0    337500.0      31086.0   \n",
       "7734               Y             1           67500.0    135000.0       9018.0   \n",
       "2174               Y             0           67500.0    202500.0      10125.0   \n",
       "\n",
       "      ...  AMT_REQ_CREDIT_BUREAU_DAY AMT_REQ_CREDIT_BUREAU_WEEK  \\\n",
       "6444  ...                        0.0                        0.0   \n",
       "3586  ...                        0.0                        0.0   \n",
       "9349  ...                        0.0                        0.0   \n",
       "7734  ...                        NaN                        NaN   \n",
       "2174  ...                        0.0                        0.0   \n",
       "\n",
       "     AMT_REQ_CREDIT_BUREAU_MON AMT_REQ_CREDIT_BUREAU_QRT  \\\n",
       "6444                       0.0                       1.0   \n",
       "3586                       0.0                       0.0   \n",
       "9349                       0.0                       0.0   \n",
       "7734                       NaN                       NaN   \n",
       "2174                       0.0                       0.0   \n",
       "\n",
       "     AMT_REQ_CREDIT_BUREAU_YEAR  BIRTH_DATE    EMP_DATE  constant  allnan  \\\n",
       "6444                        0.0  1985-06-28  2012-06-21         1     NaN   \n",
       "3586                        3.0  1953-12-27  2018-01-01         1     NaN   \n",
       "9349                        2.0  1975-06-21  2016-06-17         1     NaN   \n",
       "7734                        NaN  1988-04-27  2009-06-05         1     NaN   \n",
       "2174                        2.0  1975-06-13  1997-01-22         1     NaN   \n",
       "\n",
       "      report_dt  \n",
       "6444 2018-01-01  \n",
       "3586 2018-01-01  \n",
       "9349 2018-01-01  \n",
       "7734 2018-01-01  \n",
       "2174 2018-01-01  \n",
       "\n",
       "[5 rows x 125 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AutoML creation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<img src=\"../../imgs/tutorial_1_pipeline.png\" alt=\"AutoML pipeline for this task\" style=\"width:100%;\"/>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 1. Create Task and PandasReader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 4.03 ms, sys: 25 µs, total: 4.05 ms\n",
      "Wall time: 2.99 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "task = Task('binary')\n",
    "reader = PandasToPandasReader(task, cv=N_FOLDS, random_state=RANDOM_STATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 2. Create feature selector (if necessary) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Copying TaskTimer may affect the parent PipelineTimer, so copy will create new unlimited TaskTimer\n",
      "CPU times: user 0 ns, sys: 1.91 ms, total: 1.91 ms\n",
      "Wall time: 1.56 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "model0 = BoostLGBM(\n",
    "    default_params={'learning_rate': 0.05, 'num_leaves': 64, 'seed': 42, 'num_threads': N_THREADS}\n",
    ")\n",
    "pipe0 = LGBSimpleFeatures()\n",
    "mbie = ModelBasedImportanceEstimator()\n",
    "selector = ImportanceCutoffSelector(pipe0, model0, mbie, cutoff=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3.1. Create 1st level ML pipeline for AutoML "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our first level ML pipeline:\n",
    "- Simple features for gradient boosting built on selected features (using step 2) \n",
    "- 2 different models:\n",
    "    * LightGBM with params tuning (using OptunaTuner)\n",
    "    * LightGBM with heuristic params\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 51 µs, sys: 37 µs, total: 88 µs\n",
      "Wall time: 96.8 µs\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "pipe = LGBSimpleFeatures()\n",
    "\n",
    "params_tuner1 = OptunaTuner(n_trials=20, timeout=30) # stop after 20 iterations or after 30 seconds \n",
    "model1 = BoostLGBM(\n",
    "    default_params={'learning_rate': 0.05, 'num_leaves': 128, 'seed': 1, 'num_threads': N_THREADS}\n",
    ")\n",
    "model2 = BoostLGBM(\n",
    "    default_params={'learning_rate': 0.025, 'num_leaves': 64, 'seed': 2, 'num_threads': N_THREADS}\n",
    ")\n",
    "\n",
    "pipeline_lvl1 = MLPipeline([\n",
    "    (model1, params_tuner1),\n",
    "    model2\n",
    "], pre_selection=selector, features_pipeline=pipe, post_selection=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 3.2. Create 2nd level ML pipeline for AutoML "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our second level ML pipeline:\n",
    "- Using simple features as well, but now it will be Out-Of-Fold (OOF) predictions of algos from 1st level\n",
    "- Only one LGBM model without params tuning\n",
    "- Without feature selection on this stage because we want to use all OOFs here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 41 µs, sys: 29 µs, total: 70 µs\n",
      "Wall time: 81.5 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "pipe1 = LGBSimpleFeatures()\n",
    "\n",
    "model = BoostLGBM(\n",
    "    default_params={'learning_rate': 0.05, 'num_leaves': 64, 'max_bin': 1024, 'seed': 3, 'num_threads': N_THREADS},\n",
    "    freeze_defaults=True\n",
    ")\n",
    "\n",
    "pipeline_lvl2 = MLPipeline([model], pre_selection=None, features_pipeline=pipe1, post_selection=None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 4. Create AutoML pipeline "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "AutoML pipeline consist of:\n",
    "- Reader for data preparation\n",
    "- First level ML pipeline (as built in step 3.1)\n",
    "- Second level ML pipeline (as built in step 3.2)\n",
    "- `Skip_conn = False` equals here \"not to use initial features on the second level pipeline\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 35 µs, sys: 24 µs, total: 59 µs\n",
      "Wall time: 73.7 µs\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "automl = AutoML(reader, [\n",
    "    [pipeline_lvl1],\n",
    "    [pipeline_lvl2],\n",
    "], skip_conn=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 5. Train AutoML on loaded data "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In cell below we train AutoML with target column `TARGET` to receive fitted model and OOF predictions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[LightGBM] [Warning] seed is set=42, random_state=42 will be ignored. Current value: seed=42\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=1, random_state=42 will be ignored. Current value: seed=1\n",
      "[LightGBM] [Warning] seed is set=2, random_state=42 will be ignored. Current value: seed=2\n",
      "[LightGBM] [Warning] seed is set=2, random_state=42 will be ignored. Current value: seed=2\n",
      "[LightGBM] [Warning] seed is set=2, random_state=42 will be ignored. Current value: seed=2\n",
      "[LightGBM] [Warning] seed is set=2, random_state=42 will be ignored. Current value: seed=2\n",
      "[LightGBM] [Warning] seed is set=2, random_state=42 will be ignored. Current value: seed=2\n",
      "[LightGBM] [Warning] seed is set=3, random_state=42 will be ignored. Current value: seed=3\n",
      "[LightGBM] [Warning] seed is set=3, random_state=42 will be ignored. Current value: seed=3\n",
      "[LightGBM] [Warning] seed is set=3, random_state=42 will be ignored. Current value: seed=3\n",
      "[LightGBM] [Warning] seed is set=3, random_state=42 will be ignored. Current value: seed=3\n",
      "[LightGBM] [Warning] seed is set=3, random_state=42 will be ignored. Current value: seed=3\n",
      "oof_pred:\n",
      "array([[0.07027727],\n",
      "       [0.06983411],\n",
      "       [0.06983411],\n",
      "       ...,\n",
      "       [0.04349083],\n",
      "       [0.09716105],\n",
      "       [0.12494681]], dtype=float32)\n",
      "Shape = (8000, 1)\n",
      "CPU times: user 4min 23s, sys: 2.63 s, total: 4min 26s\n",
      "Wall time: 37.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "\n",
    "oof_pred = automl.fit_predict(train_data, roles={'target': TARGET_NAME})\n",
    "print('oof_pred:\\n{}\\nShape = {}'.format(oof_pred, oof_pred.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 6. Analyze fitted model  "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below we analyze feature importances of different algos:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Feature importances of selector:\n",
      "EXT_SOURCE_3              1029.681686\n",
      "EXT_SOURCE_2               894.265428\n",
      "BIRTH_DATE                 537.081401\n",
      "EXT_SOURCE_1               424.764621\n",
      "DAYS_LAST_PHONE_CHANGE     262.583100\n",
      "                             ...     \n",
      "FLAG_DOCUMENT_16             0.000000\n",
      "FLAG_DOCUMENT_14             0.000000\n",
      "FLAG_DOCUMENT_13             0.000000\n",
      "FLAG_DOCUMENT_11             0.000000\n",
      "FLAG_PHONE                   0.000000\n",
      "Length: 110, dtype: float64\n",
      "======================================================================\n",
      "Feature importances of top level algorithm:\n",
      "Lvl_0_Pipe_0_Mod_0_LightGBM_prediction_0    2546.473691\n",
      "Lvl_0_Pipe_0_Mod_1_LightGBM_prediction_0    1686.589227\n",
      "dtype: float64\n",
      "======================================================================\n",
      "Feature importances of lowest level algorithm - model 0:\n",
      "EXT_SOURCE_2                  1500.371550\n",
      "EXT_SOURCE_3                  1382.049802\n",
      "dtdiff__BIRTH_DATE             714.069627\n",
      "EXT_SOURCE_1                   573.079861\n",
      "DAYS_REGISTRATION              461.927863\n",
      "                                 ...     \n",
      "ord__HOUSETYPE_MODE              1.985318\n",
      "ELEVATORS_MEDI                   1.862320\n",
      "FLAG_DOCUMENT_6                  0.000000\n",
      "REG_REGION_NOT_WORK_REGION       0.000000\n",
      "ord__FLAG_OWN_CAR                0.000000\n",
      "Length: 85, dtype: float64\n",
      "======================================================================\n",
      "Feature importances of lowest level algorithm - model 1:\n",
      "EXT_SOURCE_3                   2666.270588\n",
      "EXT_SOURCE_2                   2425.430385\n",
      "dtdiff__BIRTH_DATE             1607.440484\n",
      "DAYS_REGISTRATION              1217.128893\n",
      "SK_ID_CURR                     1136.992744\n",
      "                                  ...     \n",
      "LIVE_REGION_NOT_WORK_REGION       9.561320\n",
      "ord__EMERGENCYSTATE_MODE          7.256624\n",
      "REG_REGION_NOT_WORK_REGION        5.843864\n",
      "ord__NAME_CONTRACT_TYPE           3.890026\n",
      "FLAG_DOCUMENT_6                   3.523548\n",
      "Length: 85, dtype: float64\n",
      "======================================================================\n"
     ]
    }
   ],
   "source": [
    "print('Feature importances of selector:\\n{}'\n",
    "              .format(selector.get_features_score()))\n",
    "print('=' * 70)\n",
    "\n",
    "print('Feature importances of top level algorithm:\\n{}'\n",
    "              .format(automl.levels[-1][0].ml_algos[0].get_features_score()))\n",
    "print('=' * 70)\n",
    "\n",
    "print('Feature importances of lowest level algorithm - model 0:\\n{}'\n",
    "              .format(automl.levels[0][0].ml_algos[0].get_features_score()))\n",
    "print('=' * 70)\n",
    "\n",
    "print('Feature importances of lowest level algorithm - model 1:\\n{}'\n",
    "              .format(automl.levels[0][0].ml_algos[1].get_features_score()))\n",
    "print('=' * 70)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Step 7. Predict to test data and check scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Prediction for test data:\n",
      "array([[0.060448  ],\n",
      "       [0.07832611],\n",
      "       [0.05339179],\n",
      "       ...,\n",
      "       [0.06192666],\n",
      "       [0.07732402],\n",
      "       [0.20730501]], dtype=float32)\n",
      "Shape = (2000, 1)\n",
      "Check scores...\n",
      "OOF score: 0.6979918272484156\n",
      "TEST score: 0.7158254076086956\n",
      "CPU times: user 421 ms, sys: 11.6 ms, total: 433 ms\n",
      "Wall time: 103 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "test_pred = automl.predict(test_data)\n",
    "print('Prediction for test data:\\n{}\\nShape = {}'\n",
    "              .format(test_pred, test_pred.shape))\n",
    "\n",
    "print('Check scores...')\n",
    "print('OOF score: {}'.format(roc_auc_score(train_data[TARGET_NAME].values, oof_pred.data[:, 0])))\n",
    "print('TEST score: {}'.format(roc_auc_score(test_data[TARGET_NAME].values, test_pred.data[:, 0])))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "vscode": {
   "interpreter": {
    "hash": "77738023c946bcedeeb6a5c983bcfb7849325694ebd87acbc9267f45e9f4af48"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
